{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preamble: Keras and TensorFlow tight Integration"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"imgs/tweet1.png\" width=\"50%\" />\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Suggested Reading: \n",
    "\n",
    "* [Launch a GPU-backed Google Compute Engine instance and setup Tensorflow, Keras and Jupyter](https://hackernoon.com/launch-a-gpu-backed-google-compute-engine-instance-and-setup-tensorflow-keras-and-jupyter-902369ed5272)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"imgs/tweet2.png\" width=\"50%\" />"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.2.1'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.contrib import keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tensorboard Integration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.datasets import cifar100\n",
    "\n",
    "(X_train, Y_train), (X_test, Y_test) = cifar100.load_data(label_mode='fine')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "\n",
    "img_rows, img_cols = 32, 32\n",
    "\n",
    "if K.image_data_format() == 'channels_first':\n",
    "    shape_ord = (3, img_rows, img_cols)\n",
    "else:  # channel_last\n",
    "    shape_ord = (img_rows, img_cols, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32, 32, 3)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shape_ord"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((50000, 32, 32, 3), (10000, 32, 32, 3))"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "nb_classes = len(np.unique(Y_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.applications import vgg16\n",
    "from keras.layers import Input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 32, 32, 3)         0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 32, 32, 64)        1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 32, 32, 64)        36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 16, 16, 128)       73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 16, 16, 128)       147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 8, 8, 256)         295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 8, 8, 256)         590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 8, 8, 256)         590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 4, 4, 512)         1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 2, 2, 512)         0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 1, 1, 512)         0         \n",
      "=================================================================\n",
      "Total params: 14,714,688\n",
      "Trainable params: 14,714,688\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "vgg16_model = vgg16.VGG16(weights='imagenet', include_top=False, \n",
    "                          input_tensor=Input(shape_ord))\n",
    "vgg16_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for layer in vgg16_model.layers:\n",
    "    layer.trainable = False  # freeze layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.layers.core import Dense, Dropout, Flatten\n",
    "from keras.layers.normalization import BatchNormalization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Keras Functional APIs! \n",
    "\n",
    "Remember:\n",
    "**Valar Morghulis**: _All Layers are Callables!_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "x = Flatten(input_shape=vgg16_model.output.shape)(vgg16_model.output)\n",
    "x = Dense(4096, activation='relu', name='ft_fc1')(x)\n",
    "x = Dropout(0.5)(x)\n",
    "x = BatchNormalization()(x)\n",
    "predictions = Dense(nb_classes, activation = 'softmax')(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#create graph of your new model\n",
    "model = Model(inputs=vgg16_model.input, outputs=predictions)\n",
    "\n",
    "#compile the model\n",
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 32, 32, 3)         0         \n",
      "_________________________________________________________________\n",
      "block1_conv1 (Conv2D)        (None, 32, 32, 64)        1792      \n",
      "_________________________________________________________________\n",
      "block1_conv2 (Conv2D)        (None, 32, 32, 64)        36928     \n",
      "_________________________________________________________________\n",
      "block1_pool (MaxPooling2D)   (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "block2_conv1 (Conv2D)        (None, 16, 16, 128)       73856     \n",
      "_________________________________________________________________\n",
      "block2_conv2 (Conv2D)        (None, 16, 16, 128)       147584    \n",
      "_________________________________________________________________\n",
      "block2_pool (MaxPooling2D)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "block3_conv1 (Conv2D)        (None, 8, 8, 256)         295168    \n",
      "_________________________________________________________________\n",
      "block3_conv2 (Conv2D)        (None, 8, 8, 256)         590080    \n",
      "_________________________________________________________________\n",
      "block3_conv3 (Conv2D)        (None, 8, 8, 256)         590080    \n",
      "_________________________________________________________________\n",
      "block3_pool (MaxPooling2D)   (None, 4, 4, 256)         0         \n",
      "_________________________________________________________________\n",
      "block4_conv1 (Conv2D)        (None, 4, 4, 512)         1180160   \n",
      "_________________________________________________________________\n",
      "block4_conv2 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_conv3 (Conv2D)        (None, 4, 4, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block4_pool (MaxPooling2D)   (None, 2, 2, 512)         0         \n",
      "_________________________________________________________________\n",
      "block5_conv1 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv2 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_conv3 (Conv2D)        (None, 2, 2, 512)         2359808   \n",
      "_________________________________________________________________\n",
      "block5_pool (MaxPooling2D)   (None, 1, 1, 512)         0         \n",
      "_________________________________________________________________\n",
      "flatten_1 (Flatten)          (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "ft_fc1 (Dense)               (None, 4096)              2101248   \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 4096)              0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_1 (Batch (None, 4096)              16384     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 100)               409700    \n",
      "=================================================================\n",
      "Total params: 17,242,020\n",
      "Trainable params: 2,519,140\n",
      "Non-trainable params: 14,722,880\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `TensorBoard` Callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.callbacks import TensorBoard"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```python\n",
    "\n",
    "# Arguments\n",
    "    log_dir: the path of the directory where to save the log\n",
    "        files to be parsed by TensorBoard.\n",
    "    histogram_freq: frequency (in epochs) at which to compute activation\n",
    "        and weight histograms for the layers of the model. If set to 0,\n",
    "        histograms won't be computed. Validation data (or split) must be\n",
    "        specified for histogram visualizations.\n",
    "    write_graph: whether to visualize the graph in TensorBoard.\n",
    "        The log file can become quite large when\n",
    "        write_graph is set to True.\n",
    "    write_grads: whether to visualize gradient histograms in TensorBoard.\n",
    "        `histogram_freq` must be greater than 0.\n",
    "    write_images: whether to write model weights to visualize as\n",
    "        image in TensorBoard.\n",
    "    embeddings_freq: frequency (in epochs) at which selected embedding\n",
    "        layers will be saved.\n",
    "    embeddings_layer_names: a list of names of layers to keep eye on. If\n",
    "        None or empty list all the embedding layer will be watched.\n",
    "    embeddings_metadata: a dictionary which maps layer name to a file name\n",
    "        in which metadata for this embedding layer is saved. \n",
    "```\n",
    "\n",
    "See the [details](https://www.tensorflow.org/how_tos/embedding_viz/#metadata_optional)\n",
    "about metadata files format. In case if the same metadata file is used for all embedding layers, string can be passed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "## one-hot Encoding of labels (1 to 100 classes)\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Y_train = np_utils.to_categorical(Y_train)\n",
    "Y_test = np_utils.to_categorical(Y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((50000, 100), (10000, 100))"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Y_train.shape, Y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "781/781 [==============================] - 119s - loss: 0.0157 - acc: 0.9975 - val_loss: 7.2698 - val_acc: 0.0620\n",
      "Epoch 2/10\n",
      "781/781 [==============================] - 55s - loss: 1.1926e-07 - acc: 1.0000 - val_loss: 7.4186 - val_acc: 0.0616\n",
      "Epoch 3/10\n",
      "781/781 [==============================] - 54s - loss: 1.1922e-07 - acc: 1.0000 - val_loss: 7.4713 - val_acc: 0.0623\n",
      "Epoch 4/10\n",
      "781/781 [==============================] - 54s - loss: 1.1921e-07 - acc: 1.0000 - val_loss: 7.4838 - val_acc: 0.0617\n",
      "Epoch 5/10\n",
      "781/781 [==============================] - 54s - loss: 1.1922e-07 - acc: 1.0000 - val_loss: 7.5028 - val_acc: 0.0616\n",
      "Epoch 6/10\n",
      "781/781 [==============================] - 54s - loss: 1.1921e-07 - acc: 1.0000 - val_loss: 7.5235 - val_acc: 0.0616\n",
      "Epoch 7/10\n",
      "781/781 [==============================] - 54s - loss: 1.1921e-07 - acc: 1.0000 - val_loss: 7.5361 - val_acc: 0.0612\n",
      "Epoch 8/10\n",
      "781/781 [==============================] - 54s - loss: 1.1921e-07 - acc: 1.0000 - val_loss: 7.5396 - val_acc: 0.0616\n",
      "Epoch 9/10\n",
      "781/781 [==============================] - 54s - loss: 1.1921e-07 - acc: 1.0000 - val_loss: 7.5443 - val_acc: 0.0616\n",
      "Epoch 10/10\n",
      "781/781 [==============================] - 54s - loss: 1.1921e-07 - acc: 1.0000 - val_loss: 7.5478 - val_acc: 0.0618c: 1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7f1485d8d4e0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def generate_batches(X, Y, batch_size=128):\n",
    "    \"\"\"\"\"\"\n",
    "    # Iterations has to go indefinitely\n",
    "    start = 0\n",
    "    while True:\n",
    "        yield (X[start:start+batch_size], Y[start:start+batch_size])\n",
    "        start=batch_size\n",
    "        \n",
    "# Get a subset of Validation Data (for Speed purposes) - 10%\n",
    "X_val, Y_val = X_test[:1000], Y_test[:1000]\n",
    "\n",
    "batch_size = 64\n",
    "train_steps_per_epoch = np.floor(X_train.shape[0] / batch_size)\n",
    "valid_steps_per_epoch = np.floor(X_test.shape[0] / batch_size)\n",
    "model.fit_generator(generate_batches(X_train, Y_train, batch_size=batch_size), steps_per_epoch=train_steps_per_epoch, \n",
    "                    validation_data=(X_test, Y_test), epochs=10, verbose=1,\n",
    "                    callbacks=[TensorBoard(log_dir='./tf_logs', histogram_freq=10, \n",
    "                                           write_graph=True, write_images=True, \n",
    "                                           embeddings_freq=10, \n",
    "                                           embeddings_layer_names=['block1_conv2', \n",
    "                                                                   'block5_conv1', \n",
    "                                                                   'ft_fc1'], \n",
    "                                           embeddings_metadata=None)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Runing Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process is terminated.\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "python -m tensorflow.tensorboard --logdir=./tf_logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
